In [1]:
drive_path = 'c:/'
import numpy as np
import pandas as pd
import os
import sys
import matplotlib.pyplot as plt
%matplotlib inline
from scipy.stats import ks_2samp
from scipy.stats import anderson_ksamp
from scipy.stats import kruskal
from scipy.stats import variation
from scipy import signal as sps
import seaborn as sns
import glob
import re

Detrended peaks


In [3]:
# (pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160330_3\\44dtnew.txt'))
# badfile=(pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160330_3\\44dt.txt'))
# rightfile=badfile.iloc[:,1:16]
# rightfile
# rightfile.to_csv('44dtnew.txt')

In [2]:
date='160525'
os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\HabituationFiles\\%s'%date)

In [3]:
#This piece spits out all the peaks from one session in one dataframe
peakdf=pd.DataFrame([])
# date='160626_2'
# os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\%s'%date)
for filename in glob.glob('*dt.txt'):
    f=pd.read_csv(filename,nrows=175)
    df=f[[col for col in f.columns if 'G PMT' in col]]
    peak=[]
    for col in df.columns:
        a=df[col]
        firsta=1;
        firstb=24;
        #Figures out if there is a min or max and sees if it passes threshold (3SD)
        if np.absolute(min(a[26:80]))>np.absolute(max(a[26:80])) and np.absolute(min(a[26:80]))>=3*np.std(df[col][firsta:firstb]):
            b=min(a[26:80])
            peak.append(b)
        elif np.absolute(max(a[26:80]))>np.absolute(min(a[26:80]))and np.absolute(max(a[26:80]))>=3*np.std(df[col][firsta:firstb]):
            b=max(a[26:80])
            peak.append(b)
        else:
            b=0
            peak.append(b)
        peaks=pd.DataFrame(peak).T
    peaks.columns=df.columns
    peaks=pd.concat([pd.DataFrame({'Trial':[int(filename.split('dt')[0])]}),peaks],axis=1)
    peakdf=peakdf.append(peaks,ignore_index=True)
# peakdf.to_csv('%s_peaks.csv'%date,index=False)

In [4]:
peakdf


Out[4]:
Trial G PMT (1) G PMT (2) G PMT (3) G PMT (4) G PMT (5) G PMT (6) G PMT (7) G PMT (8) G PMT (9) G PMT (10) G PMT (11) G PMT (12) G PMT (13)
0 10 0.145998 0.096714 0.000000 0.000000 0.212776 -0.099937 -0.152917 0.104205 0.142475 0.178942 0.140556 0.000000 0.000000
1 11 0.000000 -0.139088 0.000000 -0.117330 0.000000 -0.107426 -0.117268 0.000000 0.183443 0.160654 0.000000 0.000000 0.000000
2 12 0.196610 0.000000 -0.132449 -0.103636 -0.094150 0.145195 0.000000 0.094650 0.101310 -0.113630 0.107963 0.000000 -0.094127
3 1 0.000000 -0.133856 0.204637 0.310489 0.233991 0.253596 0.169331 0.171525 0.396012 0.410870 0.313415 0.000000 0.000000
4 2 -0.165721 -0.186104 0.190353 0.180027 0.304799 0.126754 -0.142533 0.156911 0.335000 0.360997 0.267152 0.082926 -0.117719
5 3 0.000000 -0.137257 0.000000 -0.143221 0.235162 0.078413 0.000000 -0.105345 0.231656 0.259872 0.149155 -0.109297 0.000000
6 4 0.000000 0.000000 0.000000 0.000000 0.234087 0.116251 0.171533 0.124724 0.338896 0.306551 0.191580 0.000000 0.000000
7 5 0.000000 -0.140312 -0.155544 -0.127843 0.149841 0.000000 -0.183504 0.000000 0.127569 0.000000 0.133320 -0.084176 -0.111516
8 6 -0.143535 0.000000 -0.325578 -0.151684 -0.103476 0.000000 -0.099074 -0.096674 0.104204 -0.099315 -0.107338 0.000000 -0.115561
9 7 0.232272 -0.118011 -0.264302 -0.150719 0.180547 0.000000 -0.180552 0.162850 0.271306 0.193235 0.239245 0.000000 -0.116297
10 8 0.000000 0.000000 0.158278 -0.133725 0.189298 -0.097343 -0.153193 0.000000 0.116627 0.105091 0.000000 0.000000 0.000000
11 9 0.195681 0.000000 0.000000 0.225418 0.291858 0.135967 0.000000 0.181339 0.156053 0.143911 0.157959 0.000000 0.000000

In [8]:
trials=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Analysis\\Odor_Panel\\Odor_Trials.csv')
filerow=trials.loc[trials['File']==date]
odortrials={}
for t in filerow.Odor.unique():
    y={t:[int(x) for x in filerow.loc[filerow['Odor'] == t][['T1','T2','T3','T4']].values.tolist()[0]]}
    odortrials.update(y)

In [9]:
odortrials


Out[9]:
{'Post30': [9, 10, 11, 12], 'Post5': [5, 6, 7, 8], 'Pre': [1, 2, 3, 4]}

In [ ]:


In [ ]:


In [10]:
#Get average peak across all trials using peakdf dataframe
meandf=pd.DataFrame([])
for key in odortrials:
    odor=odortrials[key]
    mean=[]
    for col in peakdf.loc[peakdf['Trial'].isin(odor)][[col for col in peakdf.loc[peakdf['Trial'].isin(odor)].columns if 'G PMT' in col]]:
        mean.append(peakdf.loc[peakdf['Trial'].isin(odor)][col].mean())
    mean=pd.DataFrame(mean).T
    mean.columns=peakdf.loc[peakdf['Trial'].isin(odor)][[col for col in peakdf.loc[peakdf['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
    meandf=meandf.append(mean)
meandf=meandf.reset_index(drop=True)
meandf.columns=[str(col)+'_'+date for col in meandf.columns]
meandf=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meandf],axis=1)
# meandf.to_csv('%s_mean.csv'%date,index=False)

In [11]:
meandf


Out[11]:
Odor G PMT (1)_160525 G PMT (2)_160525 G PMT (3)_160525 G PMT (4)_160525 G PMT (5)_160525 G PMT (6)_160525 G PMT (7)_160525 G PMT (8)_160525 G PMT (9)_160525 G PMT (10)_160525 G PMT (11)_160525 G PMT (12)_160525 G PMT (13)_160525
0 Pre -0.041430 -0.114304 0.098747 0.086824 0.252010 0.143753 0.049583 0.086954 0.325391 0.334573 0.230325 -0.006593 -0.029430
1 Post5 0.022184 -0.064581 -0.146787 -0.140993 0.104053 -0.024336 -0.154081 0.016544 0.154927 0.049753 0.066307 -0.021044 -0.085843
2 Post30 0.134572 -0.010593 -0.033112 0.001113 0.102621 0.018450 -0.067546 0.095049 0.145820 0.092469 0.101620 0.000000 -0.023532

In [7]:
#Get proportion of successful trials
successdf=pd.DataFrame([])
for key in odortrials:
    odor=odortrials[key]
    newdf=peakdf.loc[peakdf['Trial'].isin(odor)]
    s=[]
    for col in peakdf.loc[peakdf['Trial'].isin(odor)][[col for col in peakdf.loc[peakdf['Trial'].isin(odor)].columns if 'G PMT' in col]]:
        s.append(np.divide((newdf.loc[:,col]!=0).sum(),float(len(newdf.loc[:,col]))))
    s=pd.DataFrame(s).T
    s.columns=peakdf.loc[peakdf['Trial'].isin(odor)][[col for col in peakdf.loc[peakdf['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
    successdf=successdf.append(s)
successdf=successdf.reset_index(drop=True)
successdf.columns=[str(col)+'_'+date for col in successdf.columns]
successdf=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),successdf],axis=1)
successdf.to_csv('%s_success.csv'%date,index=False)

Detrended integral


In [8]:
# # date='160330_3'
# # os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\%s'%date)
# #example dataframe
# for filename in 
# df=pd.read_csv('1dt.txt')
# df=df[[col for col in df.columns if 'G PMT' in col]]

In [9]:
temp=pd.DataFrame([])
date='160330_3'
os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\%s'%date)
#Pull the trials that correspond to specific date/odors
trials=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Analysis\\Odor_Panel\\Odor_Trials.csv')
filerow=trials.loc[trials['File']==date]
odortrials={}
for t in trials.Odor.unique():
    y={t:[int(x) for x in filerow.loc[filerow['Odor'] == t][['T1','T2','T3','T4']].values.tolist()[0]]}
    odortrials.update(y)
#Get the frame rate for a specified date
num=trials.File.unique().tolist().index('%s'%date)
fr=trials.loc[trials['File']==trials.File.unique().tolist()[num]]['FrameRate'].iloc[0]
#Get the integral
intdf=pd.DataFrame([])
for filename in glob.glob('*dt.txt'):
    f=pd.read_csv(filename,nrows=125)
    df=f[[col for col in f.columns if 'G PMT' in col]]
    winstart=np.int(4*fr)
    winend=np.int(12*fr)
    integral=[]
    for col in df.columns:
        a=df[col]
        firsta=1;
        firstb=24;
        #Figures out if there is a min or max and sees if it passes threshold (3SD)
        if np.absolute(min(a[26:80]))>np.absolute(max(a[26:80])) and np.absolute(min(a[26:80]))>=3*np.std(df[col][firsta:firstb]):
            b=sum(df[col][winstart:winend]*(1/fr))
            integral.append(b)
        elif np.absolute(max(a[26:80]))>np.absolute(min(a[26:80]))and np.absolute(max(a[26:80]))>=3*np.std(df[col][firsta:firstb]):
            b=sum(df[col][winstart:winend]*(1/fr))
            integral.append(b)
        else:
            b=0
            integral.append(b)
    integral=pd.DataFrame(integral).T
    integral.columns=df.columns
    integral=pd.concat([pd.DataFrame({'Trial':[int(filename.split('dt')[0])]}),integral],axis=1)
    intdf=intdf.append(integral)
# intdf.to_csv('%s_integral.csv'%date,index=False)

In [10]:
#Get average integral across all trials using integral dataframe
meanint=pd.DataFrame([])
for key in odortrials:
    odor=odortrials[key]
    mean=[]
    for col in intdf.loc[intdf['Trial'].isin(odor)][[col for col in intdf.loc[intdf['Trial'].isin(odor)].columns if 'G PMT' in col]]:
        mean.append(intdf.loc[intdf['Trial'].isin(odor)][col].mean())
    mean=pd.DataFrame(mean).T
    mean.columns=intdf.loc[intdf['Trial'].isin(odor)][[col for col in intdf.loc[intdf['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
    meanint=meanint.append(mean)
meanint=meanint.reset_index(drop=True)
meanint.columns=[str(col)+'_'+date for col in meanint.columns]
meanint=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meanint],axis=1)
# meanint.to_csv('%s_meanint.csv'%date,index=False)

In [ ]:

Detrended baseline


In [11]:
temp=pd.DataFrame([])
date='160330_3'
os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\%s'%date)
#Pull the trials that correspond to specific date/odors
trials=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Analysis\\Odor_Panel\\Odor_Trials.csv')
filerow=trials.loc[trials['File']==date]
odortrials={}
for t in trials.Odor.unique():
    y={t:[int(x) for x in filerow.loc[filerow['Odor'] == t][['T1','T2','T3','T4']].values.tolist()[0]]}
    odortrials.update(y)
#Get the frame rate for a specified date
num=trials.File.unique().tolist().index('%s'%date)
fr=trials.loc[trials['File']==trials.File.unique().tolist()[num]]['FrameRate'].iloc[0]
#Get baseline
baseline=pd.DataFrame([])
for filename in glob.glob('*dt.txt'):
    f=pd.read_csv(filename,nrows=125)
    df=f[[col for col in f.columns if 'G PMT' in col]]
    winstart=np.int(4*fr)
    winend=np.int(12*fr)
    base=[]
    for col in df.columns:
        a=df[col]
        firsta=1;
        firstb=24;
        b=(df[col][firsta:firstb]).mean()
        base.append(b)
    base=pd.DataFrame(base).T
    base.columns=df.columns
    base=pd.concat([pd.DataFrame({'Trial':[int(filename.split('dt')[0])]}),base],axis=1)
    baseline=baseline.append(base)
# baseline.to_csv('%s_baseline.csv'%date,index=False)

In [36]:
baseline


Out[36]:
Trial G PMT (1) G PMT (2) G PMT (3) G PMT (4) G PMT (5) G PMT (6) G PMT (7) G PMT (8) G PMT (9) G PMT (10) G PMT (11) G PMT (12) G PMT (13) G PMT (14) G PMT (15)
0 10 -0.001772 -0.004882 -0.007575 0.002221 -0.001303 0.000239 -0.010954 -0.008220 -0.002090 -0.005447 -0.005013 -0.005016 -0.005820 -0.005131 -0.006786
0 11 -0.007566 -0.005516 -0.010814 0.000050 -0.004390 -0.001933 -0.000676 -0.009076 0.004436 -0.004560 -0.009323 -0.011845 -0.003606 -0.003959 -0.005751
0 12 -0.008305 -0.004887 -0.004887 0.000445 -0.003743 0.002659 0.004728 -0.002949 0.003973 -0.020656 -0.003786 -0.008455 -0.003549 -0.005309 -0.004675
0 13 -0.003940 -0.003201 -0.005427 -0.003443 -0.004063 -0.004560 -0.002286 -0.004222 -0.004502 0.005844 -0.008771 -0.003226 -0.002800 -0.005256 -0.009917
0 14 -0.001257 0.001370 0.001014 -0.002701 -0.005413 -0.000309 -0.001427 0.004934 -0.000473 0.005222 -0.001874 -0.004310 -0.000015 -0.007021 -0.002977
0 15 0.000266 -0.000219 -0.007417 0.011546 -0.005948 0.002980 0.000988 -0.007289 0.000278 -0.007043 -0.007277 0.002790 -0.011454 -0.008510 0.001835
0 16 -0.007216 -0.005283 -0.008189 -0.006310 -0.003022 -0.005340 -0.007450 -0.007774 -0.010824 0.008362 -0.003605 -0.002658 -0.002769 -0.003603 -0.000631
0 17 0.004017 -0.002160 -0.002432 0.005402 -0.001335 0.000963 -0.000760 -0.007520 -0.000687 -0.001767 -0.004742 -0.002614 -0.003313 -0.011488 -0.000444
0 18 -0.005194 -0.004806 -0.002486 0.000830 -0.002064 0.005237 -0.001066 -0.004164 0.003371 0.010896 -0.003461 -0.007090 -0.002751 -0.004746 -0.003140
0 19 -0.006556 -0.003986 0.001115 -0.000171 0.003068 0.006268 0.000263 0.010030 0.009705 0.007040 -0.001674 -0.002352 0.000548 -0.002654 0.004113
0 1 -0.000548 -0.003229 -0.007990 0.008057 0.001817 -0.000990 0.006400 -0.002069 -0.005586 -0.007082 0.005082 -0.001744 -0.003040 -0.004588 -0.001103
0 20 -0.003226 -0.001769 0.000951 -0.006304 -0.004982 -0.002668 -0.006824 -0.007780 -0.003941 0.002320 -0.000038 0.001686 0.004720 0.001089 -0.004335
0 21 -0.003123 -0.007120 -0.007369 0.001572 -0.005399 0.005065 0.003034 -0.009439 -0.004662 -0.005636 -0.003045 -0.010396 -0.004013 -0.002055 -0.005253
0 22 -0.005693 -0.002492 -0.005614 -0.002852 -0.009689 -0.005739 -0.004057 -0.005413 -0.004221 -0.009950 -0.004752 -0.005474 -0.008201 -0.011283 -0.008069
0 23 -0.000415 -0.001533 -0.005875 0.001786 0.002592 -0.001922 0.003023 0.004809 0.002633 0.025158 -0.005100 -0.003201 0.004141 -0.008042 0.005407
0 24 -0.011422 -0.010489 -0.005911 0.007461 -0.002506 0.000787 -0.001549 -0.006301 -0.002905 -0.003828 -0.004709 -0.010064 -0.001879 -0.003434 0.002009
0 25 0.003252 -0.003946 -0.007668 -0.001896 0.002527 0.001671 -0.000892 -0.011283 0.000932 -0.005515 -0.000761 0.001810 -0.005277 -0.008194 -0.003299
0 26 -0.004762 -0.000084 0.004069 0.005562 0.008646 0.003976 0.001884 -0.005966 0.004124 -0.013477 0.000289 0.000910 -0.000752 -0.004691 0.001129
0 27 -0.008181 -0.000365 -0.003211 -0.011035 -0.000628 0.003523 -0.001007 0.006324 -0.004526 -0.006977 -0.003957 -0.008099 -0.000831 -0.004394 -0.004806
0 28 0.003325 0.000202 -0.003064 -0.002425 -0.000784 0.007257 0.009028 -0.002624 0.000994 0.010180 -0.006461 0.003255 -0.005370 -0.000343 -0.000490
0 29 -0.001696 -0.000701 -0.001125 0.012517 0.001326 -0.003996 0.006354 0.005792 0.000800 -0.000505 -0.002313 0.000248 -0.002873 -0.005989 0.005428
0 2 -0.001598 -0.005472 -0.004938 0.003460 0.001921 -0.001274 0.003517 -0.004857 -0.006118 -0.012234 -0.005432 -0.008399 0.000085 -0.005428 -0.004038
0 30 -0.002833 -0.000527 -0.000287 -0.002227 0.003394 0.005831 0.010224 -0.006592 0.001601 -0.003029 -0.002808 -0.001194 0.001310 -0.005510 0.010098
0 31 -0.006510 -0.009417 -0.013408 -0.015382 -0.007518 -0.008679 0.003937 -0.004281 -0.006656 -0.006755 -0.007792 -0.006998 -0.012865 -0.006303 -0.008089
0 32 0.001032 -0.006737 -0.004866 0.009269 -0.002110 0.003508 0.010701 -0.001443 0.000121 0.003593 -0.006733 -0.000877 -0.003405 0.000217 0.001541
0 33 -0.006622 0.004089 -0.005113 -0.010059 -0.005751 -0.003447 0.001074 -0.001194 -0.005931 -0.001945 -0.000186 -0.004083 0.000808 -0.002033 -0.004086
0 34 -0.012642 -0.000813 -0.007733 -0.010615 -0.003600 -0.002286 -0.002625 0.008141 0.001538 0.002294 -0.007975 -0.006888 0.009432 -0.003565 -0.008992
0 35 -0.003551 -0.000982 0.001929 0.006260 -0.002984 0.000346 0.005185 -0.005716 0.003871 0.002387 0.001323 -0.002763 0.001254 -0.002037 0.001329
0 36 0.000877 -0.001687 -0.003775 -0.001944 0.003520 0.004421 0.007382 -0.004385 -0.005242 0.003577 -0.003889 -0.007858 0.002877 -0.003467 -0.000262
0 37 -0.004101 -0.001079 0.000927 -0.001532 -0.011712 -0.004070 0.010716 -0.009628 -0.007409 -0.003073 -0.004783 -0.003854 -0.001454 -0.009464 -0.005364
0 38 -0.001860 0.001281 0.001146 0.000575 -0.000272 0.008235 0.000643 -0.001367 0.001319 -0.002158 0.008152 -0.002218 -0.002291 -0.003585 -0.006883
0 39 -0.006557 -0.000035 0.000070 -0.008752 -0.000286 0.000584 0.000748 -0.002694 -0.002402 -0.006657 -0.003620 -0.003584 0.000777 -0.001075 -0.006069
0 3 -0.008982 -0.004910 -0.004493 -0.002444 -0.005244 0.001383 -0.005703 -0.003308 0.000005 0.000266 -0.010473 -0.004693 -0.004002 -0.002158 -0.004686
0 40 0.002315 0.004125 -0.005433 0.011182 0.004328 0.003215 0.015531 -0.000576 0.004839 0.001587 0.003299 -0.000050 0.003257 0.000074 -0.000484
0 41 -0.001642 -0.006792 -0.006383 0.007855 0.001323 0.003830 0.005244 -0.003098 0.000193 -0.007095 -0.003017 0.001018 -0.002341 -0.007493 0.000245
0 42 -0.006276 0.007560 -0.001286 0.006286 0.003473 0.005266 0.010847 0.000976 0.007173 -0.000513 0.005616 0.000226 -0.002098 0.003124 0.002960
0 43 0.005585 -0.001693 -0.002829 -0.000018 0.000338 0.006069 0.003757 0.002604 0.009451 0.003984 0.002320 0.000107 -0.004751 -0.000743 0.003203
0 44 0.005585 -0.001693 -0.002829 -0.000018 0.000338 0.006069 0.003757 0.002604 0.009451 0.003984 0.002320 0.000107 -0.004751 -0.000743 0.003203
0 45 -0.001405 -0.007417 -0.012930 0.002121 0.002611 0.006505 0.005249 0.000399 0.007072 -0.001000 0.003029 -0.002666 -0.006843 -0.006954 -0.000084
0 46 0.000116 -0.000802 -0.001841 0.001516 0.000249 0.003472 0.009513 -0.003021 -0.000497 -0.006059 -0.002298 -0.004778 0.000789 0.002662 -0.006244
0 47 0.000718 0.000458 0.001575 0.005708 0.003752 0.005167 0.014328 -0.001175 0.001750 0.002912 -0.004344 -0.001370 -0.001686 0.003439 0.007193
0 48 -0.000002 -0.004529 -0.001116 0.003678 0.001225 0.003619 0.005282 0.007976 0.001238 0.009330 -0.000316 -0.002659 -0.001976 -0.000323 -0.003178
0 49 -0.000937 0.002352 -0.009764 0.012109 -0.003518 0.001896 0.008730 -0.003143 0.006527 -0.003712 0.010767 0.005603 -0.014374 0.003671 0.000116
0 4 0.005624 0.002310 -0.000029 -0.003853 0.003552 0.000995 0.001362 0.006967 0.002162 0.008433 -0.004773 0.003141 0.006343 -0.006463 -0.002271
0 50 -0.006772 -0.004782 -0.002535 0.000790 -0.005647 -0.003764 -0.003757 -0.001713 -0.007046 -0.001654 -0.005895 -0.002215 0.000094 -0.002160 -0.003993
0 51 -0.001169 -0.001919 -0.008623 0.007920 0.003297 0.004908 0.011807 -0.006705 0.010325 -0.001840 0.011601 -0.000287 -0.006529 0.003866 0.004935
0 52 0.001745 0.000307 -0.000507 0.000640 -0.000221 0.005081 0.005918 -0.007441 0.001767 0.005480 0.004647 -0.000718 0.000849 -0.005314 0.006015
0 53 -0.000050 0.000589 -0.007219 0.002721 -0.000383 0.000763 0.008597 0.001794 0.001810 -0.001109 0.001997 0.000764 0.001001 0.001827 -0.006057
0 54 0.002422 0.000304 0.000006 0.015462 0.007654 0.008085 0.015165 -0.003077 0.015580 0.009980 0.004847 -0.005677 0.002948 0.001594 0.002101
0 55 0.001197 -0.005183 -0.000633 -0.005535 -0.000820 -0.002217 -0.001379 0.002380 0.000806 -0.002995 0.007179 -0.003684 0.000207 -0.004539 0.000154
0 56 -0.002023 -0.002551 -0.003301 0.007387 0.005354 -0.000663 0.009408 0.002421 0.007765 0.005610 0.006752 0.002536 0.000725 -0.001648 0.002243
0 5 -0.004849 -0.002010 -0.005010 0.009085 0.001185 0.001580 -0.006122 0.006492 0.002737 0.005525 -0.004817 -0.006597 -0.001290 -0.009422 -0.002597
0 6 -0.014006 -0.006061 -0.006660 0.003789 -0.004020 -0.001192 -0.005932 -0.012626 -0.011423 -0.000533 -0.005519 -0.007892 -0.003046 -0.008400 -0.009955
0 7 -0.009381 -0.003389 0.004302 -0.000447 0.004920 -0.001700 -0.007191 0.000399 -0.001335 0.007693 -0.000794 -0.011017 0.001215 0.003091 0.000993
0 8 -0.004276 -0.004025 -0.004309 0.005176 -0.000235 -0.000809 -0.001495 -0.004400 0.005616 -0.000475 -0.002123 -0.004217 -0.000217 0.002156 -0.001153
0 9 -0.002100 -0.000767 -0.000750 -0.004987 0.002989 -0.000570 0.006678 0.000698 -0.002124 0.004188 -0.003346 0.001302 -0.003353 -0.006259 -0.000741

In [12]:
#mean baseline
meanbase=pd.DataFrame([])
for key in odortrials:
    odor=odortrials[key]
    mean=[]
    for col in baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]]:
        mean.append(baseline.loc[baseline['Trial'].isin(odor)][col].mean())
    mean=pd.DataFrame(mean).T
    mean.columns=baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
    meanbase=meanbase.append(mean)
meanbase=meanbase.reset_index(drop=True)
meanbase.columns=[str(col)+'_'+date for col in meanbase.columns]
meanbase=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meanbase],axis=1)
# meanbase.to_csv('%s_meanbase.csv'%date,index=False)

In [38]:
meanbase


Out[38]:
Odor G PMT (1)_160330_3 G PMT (2)_160330_3 G PMT (3)_160330_3 G PMT (4)_160330_3 G PMT (5)_160330_3 G PMT (6)_160330_3 G PMT (7)_160330_3 G PMT (8)_160330_3 G PMT (9)_160330_3 G PMT (10)_160330_3 G PMT (11)_160330_3 G PMT (12)_160330_3 G PMT (13)_160330_3 G PMT (14)_160330_3 G PMT (15)_160330_3
0 IAA10 -0.004230 0.000723 -0.000525 0.006939 0.004007 0.004996 0.008156 -0.002754 0.007712 -0.006166 0.001741 -0.003249 -0.000863 -0.001321 0.000379
1 IAA01 0.000210 -0.002284 -0.002584 0.005723 0.000540 -0.000209 0.004656 -0.002402 -0.001625 -0.005141 -0.003696 -0.003886 -0.001328 -0.005061 -0.001324
2 AP -0.006349 -0.002911 -0.000479 0.000938 -0.003350 -0.002714 -0.002455 -0.003111 -0.002183 -0.000381 -0.002530 -0.005367 -0.001410 -0.003097 -0.002435
3 MS10 -0.003141 -0.003885 -0.003913 -0.003040 -0.001047 0.000144 0.000143 0.000346 -0.002007 -0.002806 -0.002141 -0.003498 -0.001442 -0.005421 -0.003581
4 IAA05 -0.001864 -0.002353 -0.005286 0.002599 -0.001782 -0.000211 0.003648 -0.004936 0.000865 0.003699 -0.000824 -0.003090 -0.002415 -0.004693 -0.000452
5 Hexanal01 -0.005869 -0.003338 -0.003011 -0.000483 0.000115 0.001698 0.000356 0.005437 0.004524 0.004547 -0.003022 -0.004029 0.001947 -0.001096 -0.002302
6 EB -0.000458 -0.001112 -0.003886 0.004596 -0.000258 0.004115 0.002212 -0.001196 0.003517 -0.000141 -0.003145 -0.001224 -0.004046 -0.006046 0.003135
7 MS01 -0.004528 -0.000533 -0.001770 -0.003275 -0.003056 0.000108 0.000719 -0.003364 -0.002029 0.000888 -0.003760 -0.002115 -0.000040 0.000084 -0.001479
8 PA -0.002716 -0.004157 -0.007559 0.004096 0.000683 -0.000955 0.002639 -0.000913 -0.002134 -0.000056 0.000548 -0.001705 -0.003881 -0.005284 0.000902
9 MS05 -0.005032 -0.002676 -0.003452 -0.000889 -0.000047 0.000391 0.003618 -0.001625 -0.001405 -0.001851 -0.002419 -0.002895 -0.000864 -0.002235 -0.002714
10 Blank NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
11 Hexanone 0.000610 -0.001036 -0.005234 0.001971 -0.000461 0.003094 0.005127 -0.002500 -0.000304 0.000666 -0.000235 -0.002377 -0.002292 -0.002079 -0.001918
12 Hexanal10 0.000590 0.000787 -0.002696 0.003361 0.000871 0.002375 0.008135 0.001039 0.003282 0.005650 0.000429 0.000358 -0.000351 -0.002235 -0.000427
13 THA -0.005031 -0.005494 -0.006346 -0.002695 -0.003316 0.000359 0.000174 -0.004617 -0.001314 0.001898 -0.003613 -0.005469 -0.005853 -0.005048 -0.004495
14 Hexanal05 -0.001107 -0.001968 -0.004461 -0.000158 -0.000589 0.003263 0.001248 -0.007292 0.002113 -0.001688 0.000679 -0.003243 -0.002581 -0.005263 -0.002480

In [4]:
# test=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160321_1\\160321_1_baseline.csv')

In [5]:
# plt.plot(test.iloc[:,1:-1]);

In [6]:
# test2=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160321_1\\160321_1_meanbase.csv')

In [7]:
# plt.plot(test2.iloc[:,1:]);

In [8]:
# test2

Untrended baseline dataframe


In [5]:
temp=pd.DataFrame([])
date='160626_2'
os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Untrended_For_B\\%s'%date)
#Pull the trials that correspond to specific date/odors
trials=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Analysis\\Odor_Panel\\Odor_Trials.csv')
filerow=trials.loc[trials['File']==date]
odortrials={}
for t in trials.Odor.unique():
    y={t:[int(x) for x in filerow.loc[filerow['Odor'] == t][['T1','T2','T3','T4']].values.tolist()[0]]}
    odortrials.update(y)
#Get the frame rate for a specified date
num=trials.File.unique().tolist().index('%s'%date)
fr=trials.loc[trials['File']==trials.File.unique().tolist()[num]]['FrameRate'].iloc[0]
#Get baseline
baseline=pd.DataFrame([])
for filename in glob.glob('*.txt'):
    f=pd.read_table(filename,skiprows=4)
    df=f[[col for col in f.columns if 'G PMT (' in col]]
    base=[]
    for col in df.columns:
        a=df[col]
        firsta=2;
        firstb=24;
        b=(df[col][firsta:firstb]).mean()
        base.append(b)
    base=pd.DataFrame(base).T
    base.columns=df.columns
    base=pd.concat([pd.DataFrame({'Trial':[int(filename.split('.txt')[0])]}),base],axis=1)
    baseline=baseline.append(base)
baseline.to_csv('%s_untrendedbaseline.csv'%date,index=False)

#mean baseline
meanbase=pd.DataFrame([])
for key in odortrials:
    odor=odortrials[key]
    mean=[]
    for col in baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]]:
        mean.append(baseline.loc[baseline['Trial'].isin(odor)][col].mean())
    mean=pd.DataFrame(mean).T
    mean.columns=baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
    meanbase=meanbase.append(mean)
meanbase=meanbase.reset_index(drop=True)
meanbase.columns=[str(col)+'_'+date for col in meanbase.columns]
meanbase=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meanbase],axis=1)
meanbase.to_csv('%s_meanbase.csv'%date,index=False)

In [6]:
#mean baseline
meanbase=pd.DataFrame([])
for key in odortrials:
    odor=odortrials[key]
    mean=[]
    for col in baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]]:
        mean.append(baseline.loc[baseline['Trial'].isin(odor)][col].mean())
    mean=pd.DataFrame(mean).T
    mean.columns=baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
    meanbase=meanbase.append(mean)
meanbase=meanbase.reset_index(drop=True)
meanbase.columns=[str(col)+'_'+date for col in meanbase.columns]
meanbase=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meanbase],axis=1)
# meanbase.to_csv('%s_meanbase.csv'%date,index=False)

In [8]:
meanbase


Out[8]:
Odor G PMT (1)_160626_2 G PMT (2)_160626_2 G PMT (3)_160626_2 G PMT (4)_160626_2 G PMT (5)_160626_2 G PMT (6)_160626_2 G PMT (7)_160626_2 G PMT (8)_160626_2 G PMT (9)_160626_2 ... G PMT (17)_160626_2 G PMT (18)_160626_2 G PMT (19)_160626_2 G PMT (20)_160626_2 G PMT (21)_160626_2 G PMT (22)_160626_2 G PMT (23)_160626_2 G PMT (24)_160626_2 G PMT (25)_160626_2 G PMT (26)_160626_2
0 IAA10 1291.544864 1268.901489 1050.995830 1032.657568 804.474341 911.311318 1389.563239 925.153386 1290.568693 ... 820.584477 621.505807 1068.333784 1247.801045 819.507023 776.600750 1483.688773 1220.963977 1269.520023 890.331375
1 IAA01 1281.165409 1297.232045 976.186943 1005.542795 764.650102 869.020932 1315.126239 951.213443 1572.237375 ... 799.432875 589.821364 1056.008273 1137.161398 785.458284 751.138534 1411.801398 1206.080909 1168.628364 815.871295
2 AP 1376.152591 1347.562330 1137.701045 1044.827705 822.350216 930.923795 1377.323034 947.701068 1438.613318 ... 834.238273 614.483818 1054.358159 1229.093091 803.986182 840.312534 1439.084477 1277.098977 1169.697193 833.271216
3 MS10 1387.064273 1321.578420 1128.520864 1127.331182 878.369170 1008.929523 1454.279205 930.380080 1345.281511 ... 858.798727 671.313159 1131.200716 1222.376602 839.462682 855.376091 1502.909273 1324.046932 1216.110466 924.447307
4 IAA05 1262.019511 1267.637943 999.168159 1017.357932 785.982455 896.120761 1309.384114 940.718886 1135.399284 ... 779.928852 593.644534 1040.643727 1166.757489 774.938148 750.147955 1355.461602 1162.023182 1153.194432 845.589420
5 Hexanal01 1381.141159 1334.504545 1087.204977 1064.206375 852.481955 945.278216 1507.448455 1039.246023 1766.495557 ... 861.620341 624.156148 1160.438830 1254.024864 857.482239 889.155545 1670.571489 1433.757386 1245.991182 955.639045
6 EB 1351.074568 1348.900432 1099.199443 1078.011205 836.903023 927.686307 1338.786659 949.252159 1192.446295 ... 855.680511 638.163864 1079.776602 1219.231080 864.064875 819.878080 1465.685727 1279.233636 1141.404034 836.367227
7 MS01 1221.955841 1317.361091 1068.941818 1013.127773 781.233273 892.717443 1362.300057 922.448602 1434.050114 ... 833.584830 608.652341 1035.554773 1157.286216 815.830807 800.496727 1447.919864 1231.496250 1163.552636 855.852295
8 PA 1356.900136 1295.550489 1098.453239 1001.486989 800.393045 905.167034 1431.667841 1033.042261 1351.153682 ... 821.524295 605.601545 1158.286091 1231.370250 794.850693 817.811818 1441.427114 1290.111591 1149.943580 854.777102
9 MS05 1293.685966 1277.941136 980.143170 1022.320773 829.538557 912.101295 1306.427966 923.909693 1255.346307 ... 792.268693 606.287659 1032.395341 1107.825239 838.414045 784.148602 1413.797330 1190.936477 1145.353318 822.222693
10 Blank 1069.478875 1123.839011 862.194727 958.492261 721.197125 788.247466 990.024580 614.862011 950.385307 ... 714.200580 563.431045 846.939398 828.041557 730.574841 659.146670 1134.134920 947.159659 807.150784 627.208898
11 Hexanone 1175.541159 1192.623409 984.606182 1026.571977 754.261739 855.919307 1168.203205 683.055648 1221.163852 ... 790.926443 596.212932 921.045250 933.617261 811.836273 731.421943 1263.311409 1106.326932 912.741352 732.055864
12 Hexanal10 1298.665875 1289.066727 1005.893784 1020.411682 781.169216 900.219739 1347.503795 928.040830 1261.424420 ... 827.410216 602.985364 1033.884966 1291.845386 803.198261 782.215705 1459.681011 1166.976477 1219.318307 848.670875
13 THA 1322.476625 1332.047920 1108.578761 1042.183932 825.073045 917.298011 1384.014341 1068.551398 1196.576023 ... 823.999602 615.000636 1114.208068 1161.271443 850.424841 797.764216 1541.519193 1240.643750 1094.684795 828.595068
14 Hexanal05 1274.953606 1325.420167 1026.854091 1031.134818 786.224530 884.434667 1369.621515 963.680530 1164.182788 ... 783.589606 603.474606 1065.472485 1192.553273 773.460621 773.446970 1403.306348 1208.390758 1190.651894 836.463303

15 rows × 27 columns


In [69]:
pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160325_1\\58dt.txt').size
# badfile=pd.read_table('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Untrended_For_B\\160401_1\\50.txt',skiprows=4)
# rightfile=badfile.iloc[:,3:18]
# rightfile
# rightfile.to_csv('44new.csv',index=False)


---------------------------------------------------------------------------
IOError                                   Traceback (most recent call last)
<ipython-input-69-5ce0e233d74f> in <module>()
----> 1 pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160325_1\\58dt.txt').size
      2 # badfile=pd.read_table('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Untrended_For_B\\160401_1\\50.txt',skiprows=4)
      3 # rightfile=badfile.iloc[:,3:18]
      4 # rightfile
      5 # rightfile.to_csv('44new.csv',index=False)

C:\Users\Annie\Anaconda2\lib\site-packages\pandas\io\parsers.pyc in parser_f(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skip_footer, doublequote, delim_whitespace, as_recarray, compact_ints, use_unsigned, low_memory, buffer_lines, memory_map, float_precision)
    560                     skip_blank_lines=skip_blank_lines)
    561 
--> 562         return _read(filepath_or_buffer, kwds)
    563 
    564     parser_f.__name__ = name

C:\Users\Annie\Anaconda2\lib\site-packages\pandas\io\parsers.pyc in _read(filepath_or_buffer, kwds)
    313 
    314     # Create the parser.
--> 315     parser = TextFileReader(filepath_or_buffer, **kwds)
    316 
    317     if (nrows is not None) and (chunksize is not None):

C:\Users\Annie\Anaconda2\lib\site-packages\pandas\io\parsers.pyc in __init__(self, f, engine, **kwds)
    643             self.options['has_index_names'] = kwds['has_index_names']
    644 
--> 645         self._make_engine(self.engine)
    646 
    647     def close(self):

C:\Users\Annie\Anaconda2\lib\site-packages\pandas\io\parsers.pyc in _make_engine(self, engine)
    797     def _make_engine(self, engine='c'):
    798         if engine == 'c':
--> 799             self._engine = CParserWrapper(self.f, **self.options)
    800         else:
    801             if engine == 'python':

C:\Users\Annie\Anaconda2\lib\site-packages\pandas\io\parsers.pyc in __init__(self, src, **kwds)
   1211         kwds['allow_leading_cols'] = self.index_col is not False
   1212 
-> 1213         self._reader = _parser.TextReader(src, **kwds)
   1214 
   1215         # XXX

pandas\parser.pyx in pandas.parser.TextReader.__cinit__ (pandas\parser.c:3427)()

pandas\parser.pyx in pandas.parser.TextReader._setup_parser_source (pandas\parser.c:6861)()

IOError: File C:\Users\Annie\Documents\Data\Ca_Imaging\GoodFiles\160325_1\58dt.txt does not exist

In [37]:
meanbase


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-37-c6e8063454ef> in <module>()
----> 1 meanbase

NameError: name 'meanbase' is not defined

In [ ]: